package com.spark.ml

import org.apache.spark.mllib.feature.{Word2Vec, Word2VecModel}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by TRS on 2017/6/20.
  */
object learn {

  def main(args: Array[String]): Unit = {
   // System.setProperty("hadoop.home.dir", "D:\\openSource\\hadoop-2.7.3")
    //设置本机Spark配置
    val conf = new SparkConf().setAppName("wordCount").setMaster("local")
      .set("spark.driver.memory", "2G")
    //创建Spark上下
    val sc = new SparkContext(conf)

    val input = sc.textFile("E:\\file\\res\\allType.txt").map(line => line.substring(2).split(" ").toSeq)

    val word2vec = new Word2Vec()

    val model = word2vec.fit(input)

    model.save(sc, "E:\\file\\res\\word2vec\\")
    val sameModel = Word2VecModel.load(sc, "myModelPath")

    val synonyms = model.findSynonyms("", 5)

    for((synonym, cosineSimilarity) <- synonyms) {
      println(s"$synonym $cosineSimilarity")
    }

  }

}
